package com.fengzhi.crawler.jsoup.parser;

import java.util.HashSet;
import java.util.Set;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.fengzhi.crawler.jsoup.parser.impl.BasicParser;

public class BasicParserImpl implements BasicParser {

	@Override
	public Document denoiseForDoc(Document document) {
		document.getElementsByTag("script").remove();
		document.getElementsByTag("style").remove();
		document.getElementsByTag("select").remove();
		document.getElementsByTag("link").remove();
		document.getElementsByTag("input").remove();
		document.getElementsByTag("object").remove();
		document.getElementsByTag("textarea").remove();
		// document.getElementsByTag("a").attr("href",
		// "javascript:void(0)").remove();
		document.getElementsByAttributeValue("display", "none").remove();
		document.getElementsByAttributeValueContaining("style", "display:none").remove();
		document.getElementsByAttributeValueContaining("style", "overflow: hidden").remove();

		return document;
	}

	@Override
	public Set<String> getImgUrl(Element element) {
		Set<String> imgUrl = new HashSet<String>();

		Elements imgElements = element.getElementsByTag("img");

		for (Element ele : imgElements) {
			String url = ele.attr("src");

			imgUrl.add(url);
		}

		return imgUrl;
	}

	public Set<String> getLinks(Element contentElement) {
		Elements linkElements = contentElement.getElementsByTag("a");

		if (linkElements == null) {
			return null;
		}

		Set<String> linkSet = new HashSet<String>();

		for (Element element : linkElements) {
			String url = element.attr("href");

			if (url.startsWith("#")) {
				continue;
			}

			linkSet.add(url);
		}

		return linkSet;
	}

}
